{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Holdout Method\n",
    "-In this approach, we reserve 50% of the dataset for validation and the remaining 50% for model training. However, a major disadvantage of this approach is that since we are training a model on only 50% of the dataset, there is a huge possibility that we might miss out on some interesting information about the data which will lead to a higher bias."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.7, 3. , 4.2, 1.2],\n",
       "       [6.9, 3.1, 5.4, 2.1],\n",
       "       [6.8, 2.8, 4.8, 1.4],\n",
       "       [5.2, 4.1, 1.5, 0.1],\n",
       "       [5.8, 2.7, 5.1, 1.9],\n",
       "       [4.3, 3. , 1.1, 0.1],\n",
       "       [4.8, 3.4, 1.9, 0.2],\n",
       "       [5.2, 2.7, 3.9, 1.4],\n",
       "       [4.8, 3. , 1.4, 0.3],\n",
       "       [4.9, 3.1, 1.5, 0.2],\n",
       "       [7.9, 3.8, 6.4, 2. ],\n",
       "       [5. , 2.3, 3.3, 1. ],\n",
       "       [4.6, 3.2, 1.4, 0.2],\n",
       "       [6.5, 3. , 5.5, 1.8],\n",
       "       [4.9, 3.1, 1.5, 0.1],\n",
       "       [6. , 2.2, 5. , 1.5],\n",
       "       [5.5, 2.6, 4.4, 1.2],\n",
       "       [5.8, 4. , 1.2, 0.2],\n",
       "       [5.4, 3.9, 1.3, 0.4],\n",
       "       [6.4, 2.7, 5.3, 1.9],\n",
       "       [6. , 3.4, 4.5, 1.6],\n",
       "       [5.5, 2.5, 4. , 1.3],\n",
       "       [5.5, 4.2, 1.4, 0.2],\n",
       "       [4.7, 3.2, 1.6, 0.2],\n",
       "       [6.9, 3.2, 5.7, 2.3],\n",
       "       [6. , 2.9, 4.5, 1.5],\n",
       "       [6.1, 3. , 4.6, 1.4],\n",
       "       [4.6, 3.1, 1.5, 0.2],\n",
       "       [5.7, 2.8, 4.5, 1.3],\n",
       "       [6. , 3. , 4.8, 1.8],\n",
       "       [5.8, 2.7, 4.1, 1. ],\n",
       "       [4.8, 3.4, 1.6, 0.2],\n",
       "       [6. , 2.2, 4. , 1. ],\n",
       "       [6.4, 3.1, 5.5, 1.8],\n",
       "       [6.7, 2.5, 5.8, 1.8],\n",
       "       [6.3, 3.3, 6. , 2.5],\n",
       "       [6.8, 3.2, 5.9, 2.3],\n",
       "       [5.7, 3.8, 1.7, 0.3],\n",
       "       [5.1, 3.8, 1.5, 0.3],\n",
       "       [6. , 2.7, 5.1, 1.6],\n",
       "       [7.7, 2.8, 6.7, 2. ],\n",
       "       [4.4, 3. , 1.3, 0.2],\n",
       "       [6.2, 2.2, 4.5, 1.5],\n",
       "       [4.7, 3.2, 1.3, 0.2],\n",
       "       [5.4, 3.4, 1.5, 0.4],\n",
       "       [6.7, 3.1, 5.6, 2.4],\n",
       "       [6.4, 3.2, 4.5, 1.5],\n",
       "       [7.6, 3. , 6.6, 2.1],\n",
       "       [5.5, 3.5, 1.3, 0.2],\n",
       "       [6.5, 3.2, 5.1, 2. ],\n",
       "       [5. , 3.6, 1.4, 0.2],\n",
       "       [6.9, 3.1, 5.1, 2.3],\n",
       "       [5.1, 3.5, 1.4, 0.2],\n",
       "       [6.6, 2.9, 4.6, 1.3],\n",
       "       [5.4, 3.9, 1.7, 0.4],\n",
       "       [6.3, 2.9, 5.6, 1.8],\n",
       "       [7.2, 3. , 5.8, 1.6],\n",
       "       [4.5, 2.3, 1.3, 0.3],\n",
       "       [4.9, 2.5, 4.5, 1.7],\n",
       "       [5.6, 2.8, 4.9, 2. ],\n",
       "       [7.2, 3.2, 6. , 1.8],\n",
       "       [6.7, 3.1, 4.7, 1.5],\n",
       "       [4.8, 3.1, 1.6, 0.2],\n",
       "       [6.7, 3.1, 4.4, 1.4],\n",
       "       [5.1, 3.8, 1.9, 0.4],\n",
       "       [5.2, 3.5, 1.5, 0.2],\n",
       "       [5.5, 2.4, 3.8, 1.1],\n",
       "       [5.7, 2.5, 5. , 2. ],\n",
       "       [5. , 3.4, 1.5, 0.2],\n",
       "       [6.8, 3. , 5.5, 2.1],\n",
       "       [4.4, 2.9, 1.4, 0.2],\n",
       "       [6.1, 2.8, 4.7, 1.2],\n",
       "       [6.7, 3.3, 5.7, 2.5],\n",
       "       [7.7, 2.6, 6.9, 2.3],\n",
       "       [5.7, 2.8, 4.1, 1.3]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The Hold-Out Method\n",
    "train, validation = train_test_split(data.data, test_size=0.50, random_state = 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Leave one out cross validation (LOOCV)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-In this approach, we reserve only one data point from the available dataset, and train the model on the rest of the data. This process iterates for each data point. This also has its own advantages and disadvantages. Let’s look at them:\n",
    "\n",
    "> We make use of all data points, hence the bias will be low\n",
    "\n",
    "> We repeat the cross validation process $n$ times (where $n$ is number of data points) which results in a higher execution time\n",
    "\n",
    "> This approach leads to higher variation in testing model effectiveness because we test against one data point. So, our estimation gets highly influenced by the data point. If the data point turns out to be an outlier, it can lead to a higher variation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: [1] validation: [0]\n",
      "train: [0] validation: [1]\n"
     ]
    }
   ],
   "source": [
    "# The Leave-One-Out Method\n",
    "from sklearn.model_selection import LeaveOneOut\n",
    "X = np.array([[1, 2], [3, 4]])\n",
    "y = np.array([1, 2])\n",
    "loo = LeaveOneOut()\n",
    "loo.get_n_splits(X)\n",
    "\n",
    "for train_index, val_index in loo.split(X):\n",
    "        print(\"train:\", train_index, \"validation:\", val_index)\n",
    "        X_train, X_test = X[train_index], X[val_index]\n",
    "        y_train, y_test = y[train_index], y[val_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- LOOCV leaves one data point out. Similarly, you could leave $p$ training examples out to have validation set of size $p$ for each iteration. This is called LPOCV (Leave P Out Cross Validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## k-fold cross validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above two validation methods, we’ve learnt:\n",
    "\n",
    "1) We should train the model on a large portion of the dataset. Otherwise we’ll fail to read and recognise the underlying trend in the data. This will eventually result in a higher bias\n",
    "\n",
    "2) We also need a good ratio of testing data points. As we have seen above, less amount of data points can lead to a variance error while testing the effectiveness of the model\n",
    "    \n",
    "3) We should iterate on the training and testing process multiple times. We should change the train and test dataset distribution. This helps in validating the model effectiveness properly\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do we have a method which takes care of all these 3 requirements?\n",
    "\n",
    "Yes! That method is known as “k-fold cross validation”. It’s easy to follow and implement. Below are the steps for it:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   1) Randomly split your entire dataset into $k$”folds”\n",
    "    \n",
    "   2) For each $k$-fold in your dataset, build your model on $k-1$ folds of the dataset. Then, test the model to check the effectiveness for $k$th fold\n",
    "    \n",
    "   3) Record the error you see on each of the predictions\n",
    "    \n",
    "   4) Repeat this until each of the $k$-folds has served as the test set\n",
    "    \n",
    "   5) The average of your $k$ recorded errors is called the cross-validation error and will serve as your performance metric for the model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, one of most commonly asked questions is, “How to choose the right value of $k$?”.\n",
    "\n",
    "Always remember, a lower value of $k$ is more biased, and hence undesirable. On the other hand, a higher value of $k$ is less biased, but can suffer from large variability. It is important to know that a smaller value of $k$ always takes us towards validation set approach, whereas a higher value of $k$ leads to LOOCV approach.\n",
    "\n",
    "Precisely, LOOCV is equivalent to $n$-fold cross validation where $n$ is the number of training examples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n"
     ]
    }
   ],
   "source": [
    "# K-fold Cross Validation\n",
    "from sklearn.model_selection import KFold \n",
    "kf = KFold(n_splits=2, random_state=None) \n",
    "\n",
    "for train_index, val_index in kf.split(X):\n",
    "      print(\"Train:\", train_index, \"Validation:\",val_index)\n",
    "      X_train, X_test = X[train_index], X[val_index] \n",
    "      y_train, y_test = y[train_index], y[val_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratified k-fold cross validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-Stratification is the process of rearranging the data so as to ensure that each fold is a good representative of the whole. For example, in a binary classification problem where each class comprises of 50% of the data, it is best to arrange the data such that in every fold, each class comprises of about half the instances."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-It is generally a better approach when dealing with both bias and variance. A randomly selected fold might not adequately represent the minor class, particularly in cases where there is a huge class imbalance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: [1 3] Validation: [1]\n",
      "Train: [0 2] Validation: [1]\n"
     ]
    }
   ],
   "source": [
    "# Stratified K-fold Cross Validation\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])\n",
    "y = np.array([0, 0, 1, 1])\n",
    "skf = StratifiedKFold(n_splits=2, random_state=None)\n",
    "#skf.get_n_splits(X, y)\n",
    "\n",
    "for train_index, test_index in skf.split(X, y):\n",
    "   print(\"Train:\", train_index, \"Validation:\", val_index)\n",
    "   X_train, X_test = X[train_index], X[val_index]\n",
    "   y_train, y_test = y[train_index], y[val_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Having said that, if the train set does not adequately represent the entire population, then using a stratified k-fold might not be the best idea. In such cases, one should use a simple k-fold cross validation with repetition.\n",
    "\n",
    "In repeated cross-validation, the cross-validation procedure is repeated n times, yielding n random partitions of the original sample. The n results are again averaged (or otherwise combined) to produce a single estimation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n",
      "Train: [1] Validation: [0]\n",
      "Train: [0] Validation: [1]\n"
     ]
    }
   ],
   "source": [
    "#Repeated K-Fold Cross Validation\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "rkf = RepeatedKFold(n_splits=2, n_repeats=10, random_state=None)\n",
    "# X is the feature set and y is the target\n",
    "for train_index, val_index in rkf.split(X):\n",
    "     print(\"Train:\", train_index, \"Validation:\", val_index)\n",
    "     X_train, X_test = X[train_index], X[val_index]\n",
    "     y_train, y_test = y[train_index], y[val_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References \n",
    "\n",
    "(https://www.analyticsvidhya.com/blog/2018/05/improve-model-performance-cross-validation-in-python-r/)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
